# coding: utf-8

import requests
import json
import time
import chardet
from bs4 import BeautifulSoup

#省
def do_first(url, cls):
    #headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"}
    headers = {'user-agent': 'my-app/0.0.1'}
        
    try:
        result = requests.get(url, headers = headers, timeout = 3)
    except:
        return do_second(url, cls)
        
    content = result.content.decode('gb18030')
        
    #字典
    dict = {}
    
    #解析
    soup = BeautifulSoup(content, 'lxml')
    #p_tr = soup.select('.provincetr')
    p_trs = soup.find_all("tr", class_=cls)
    for p_tr in p_trs:
        p_as = p_tr.find_all('a')
        if p_as:
            for p_a in p_as:
                p_a_text = p_a.text
                p_a_link = p_a['href']
                dict[p_a_text] = p_a_link
                #print(p_a.encode('gb18030'))
                #print(p_a_link)

    
    #print(dict)
    return dict

#市/区/街道
def do_second(url, cls):
    #headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"}
    headers = {'user-agent': 'my-app/0.0.1'}
    
    try:
        result = requests.get(url, headers = headers, timeout = 3)
    except:
        return do_second(url, cls)
        
    content = result.content.decode('gb18030')

    #字典
    dict = {}

    #解析
    soup = BeautifulSoup(content, 'lxml')
    #p_tr = soup.select('.provincetr')
    p_trs = soup.find_all("tr", class_=cls)
    for p_tr in p_trs:
        p_as = p_tr.find_all('a')
        if p_as:
            i = 0
            for p_a in p_as:
                #只取第二
                if i == 0:
                    i += 1
                    continue
                p_a_text = p_a.text
                p_a_link = p_a['href']
                dict[p_a_text] = p_a_link
                #print(p_a.encode('gb18030'))
                #print(p_a_link)
    
    #print(dict)
    return dict

if __name__ == '__main__':
    #'''
    time_start = int(time.time())
    result = {}
    url_head = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/'
    #省份
    url_province = url_head + 'index.html'
    dict_provinces = do_first(url_province, 'provincetr')
    #print(dict_provinces)
    for dict_p in dict_provinces:
        result[dict_p] = {}
        #城市
        url_province_arr = url_province.split('/')
        url_province_str = url_province.replace(url_province_arr[-1], '')
        url_city = url_province_str + dict_provinces[dict_p]
        dict_citys = do_second(url_city, 'citytr')
        #print(dict_citys)
        for dict_c in dict_citys:
            result[dict_p][dict_c] = {}
            #地区
            url_city_arr = url_city.split('/')
            url_city_str = url_city.replace(url_city_arr[-1], '')
            url_district = url_city_str + dict_citys[dict_c]
            dict_districts = do_second(url_district, 'countytr')
            #有的城市下面没有区县直接到镇
            dict_districts_len = len(dict_districts)
            if dict_districts_len == 0:
                dict_districts = do_second(url_district, 'towntr')
            #print(dict_districts)
            for dict_d in dict_districts:
                result[dict_p][dict_c][dict_d] = {}
                #街道
                url_district_arr = url_district.split('/')
                url_district_str = url_district.replace(url_district_arr[-1], '')
                if dict_districts_len == 0:
                    continue
                url_street = url_district_str + dict_districts[dict_d]
                dict_streets = do_second(url_street, 'towntr')
                for dict_s in dict_streets:
                    result[dict_p][dict_c][dict_d][dict_s] = {}
                    print(dict_s.encode('gb18030'))
                    # break
                    #print(dict_s.encode('gb18030') + ' - finshed')
                    #time.sleep(5)
                # break
                print(dict_d.encode('gb18030') + ' - finshed')
                # time.sleep(1)
            # break
            print(dict_c.encode('gb18030') + ' - finshed')
            # time.sleep(1)
        # break
        print(dict_p.encode('gb18030') + ' - finshed')
        # time.sleep(1)
        
    
    #print(result)
    #写入文件
    json_content = json.dumps(result)
    fp = open('result.txt', 'wb')
    fp.write(json_content)
    fp.close()
    
    #计算时间
    time_end = int(time.time())
    time_spend = time_end - time_start
    print('spend ' + str(time_spend) + ' secs')
    #'''
    
    #测试
    # url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/34/01/340181.html'
    # dict = do_second(url, 'towntr')
    # print(dict)